import requests
import random
from datetime import datetime
from bs4 import BeautifulSoup
from fake_useragent import UserAgent


def article_detail(link):
    response = requests.get(link, UserAgent().random)
    soup = BeautifulSoup(response.text, 'html.parser')
    div_tags = soup.select('#cnblogs_post_body')
    return div_tags


def create_time_func():
    start = datetime(2000, 1, 1)
    end = datetime(2024, 12, 31)
    create_time = start + (end - start) * random.random()
    return create_time


def blog_spider():
    print('===================================================================================')
    article_data_list = []
    '''站点与分类对应表'''
    blog_category = {1: (1, 2, 3), 2: (4, 5, 6), 3: (7, 8, 9)}
    '''标签与站点对应表'''
    blog_tags = {1: (1, 2, 3), 2: (4, 5, 6), 3: (7, 8, 9)}
    for page in range(1, 4):
        if page == 1:
            url = 'https://www.cnblogs.com/'
        else:
            url = f'https://www.cnblogs.com/#p{page}'

        response = requests.get(url, UserAgent().random)
        soup = BeautifulSoup(response.text, 'html.parser')
        id_tags = soup.select('#post_list')
        for item in id_tags:
            a_tags = item.select('a.post-item-title')
            p_tags = item.select('p.post-item-summary')
            for a, p in zip(a_tags, p_tags):
                title = a.text
                link = a.attrs.get('href')
                # 文章详细
                article_detail_data = article_detail(link)
                # 简介
                summary = p.text.strip()
                # 点赞数
                up_num = random.randint(2, 10)
                # 点踩数
                down_num = random.randint(2, 10)
                # 评论内容
                comment_num = random.randint(2, 10)
                # 博客id
                blog_id = random.randint(1, 3)
                create_time = create_time_func()
                # 分类表
                '''分类id'''
                category_id = random.choice(blog_category[blog_id])
                # 构建信息字典
                article_data_list.append({
                    'title': title,
                    'desc': summary,
                    'content': article_detail_data,
                    'up_count': up_num,
                    'down_count': down_num,
                    'comment_count': comment_num,
                    'blog_id': blog_id,
                    'category_id': category_id
                })
    for article_data in article_data_list:
        # 生成标签id
        article_blog_id = article_data.get('blog_id')
        tags_tuple = blog_tags[article_blog_id]
        # 创建数据库数据
        article_obj = Article.objects.create(**article_data)
        article_obj.create_time = create_time_func()
        # 设置标签id
        article_obj.tags.set(random.sample(tags_tuple, random.randint(1, len(tags_tuple))))
        article_obj.save()


if __name__ == '__main__':
    import os
    import django

    os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'BlogBasedSystem.settings')
    django.setup()
    '''测试代码'''
    from app_article.models import Article

    blog_spider()
